#!/usr/bin/python
# encoding=utf-8
# cosine 2010/11/26

"""镜像你的饭否
"""

__version__ = ''

import os
import sys
import urllib
from kscomm.spider.auth import Auth
from kscomm.spider.fetch import HttpFetch
from kscomm.spider.parses.htmlparse import HtmlParse
from optparse import OptionParser

g_userid = 'ycosxhack' # http://fanfou.com/ycosxhack
g_username = 'ycosxhack@126.com' # login username
g_password = '******' # login password

g_start_page = 1 # 镜像的第一页
g_end_page = 12 # 镜像的最后一页

g_login_url = 'http://fanfou.com/login' # login action
g_referer = 'http://fanfou.com/login'
g_user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12'
g_http_opener = None

# 镜像下来的js/css/img等资源保存目录
g_res_dir = 'fanfou/'

def down_media(url):
	"""下载js/css/img这类资源文件"""
	res_name = url.split('/')[-1].split('?')[0]
	res_local = g_res_dir + res_name
	if os.path.exists(res_local):
		return res_local
	urllib.urlretrieve(url,res_local)
	return res_local
	
def down_html(url,html):
	"""下载HTML"""
	res_name = url.split('/')[-1].split('?')[0]
	if not res_name.endswith(('.html','.htm')):
		res_name = res_name+'.html'
	f = open(res_name,'w')
	f.write(html)
	f.close()

def login(username,password):
	"""登录饭否"""
	global g_http_opener
	auth = Auth()
	auth.set_headers(referer=g_referer,user_agent=g_user_agent)
	auth.set_form_auth(g_login_url,{'loginname':username,'loginpass':password,'action':'login'})
	g_http_opener = auth.get_http_opener()
	
def fetch(url):
	"""获取HTML"""
	fetch = HttpFetch()
	fetch.set_http_opener(g_http_opener)
	fetch.request(url)
	html = fetch.get_html()
	return html
	
def mirror(page_url,html):
	"""镜像HTML"""
	parse = HtmlParse()
	urls_dict = parse.get_suburls(page_url,html,tags=['script','style','img'])
	script_urls = urls_dict['script']
	style_urls = urls_dict['style']
	img_urls = urls_dict['img']

	for url in script_urls:
		res_local = down_media(url)
		html = html.replace(url,res_local)
		
	for url in style_urls:
		res_local = down_media(url)
		html = html.replace(url,res_local)
	
	for url in img_urls:
		res_local = down_media(url)
		html = html.replace(url,res_local)
		
	down_html(page_url,html)
	
if __name__ == '__main__':
	usage = "python fanfou_mirror.py -u ycosxhack@126.com -p 123456 -i ycosxhack -s 1 -e 12" 
	parser = OptionParser(usage,version=__version__)
	parser.add_option("-i", "--userid", dest="userid", 
	                  help=u"饭否用户唯一的地址标志，如http://fanfou.com/ycosxhack，唯一标志为ycosxhack")
	parser.add_option("-u", "--username", dest="username", 
	                  help=u"饭否登录用户名")
	parser.add_option("-p", "--password", dest="password", 
	                  help=u"饭否登录密码")
	parser.add_option("-s", "--start", type="int", dest="start_page",default=1, 
	                  help=u"镜像开始页码，默认从第一页开始镜像")
	parser.add_option("-e", "--end", type="int", dest="end_page",default=1, 
	                  help=u"镜像结束页码，默认第一页结束")

	(options, args) = parser.parse_args()
	try:
		g_userid = options.userid
		g_username = options.username
		g_password = options.password
		g_start_page = options.start_page
		g_end_page = options.end_page
	except Exception,e:
		print e
		parser.print_help()
		sys.exit(0)
	if not g_userid or not g_username or not g_password:
		parser.print_help()
		sys.exit(1)
	
	try:
		if not os.path.exists(g_res_dir):
			os.mkdir(g_res_dir)
	except Exception,e:
		print 'mkdir %s error: %s'%(g_res_dir,e)
		sys.exit(2)
		
	page_urls = []
	page = 'http://fanfou.com/'+g_userid+'/p.'
	for i in xrange(g_start_page,g_end_page+1):
		page_urls.append(page+str(i))
		
	print '[*] login fanfou.com...'
	login(g_username,g_password)
	for page_url in page_urls:
		html = fetch(page_url)
		mirror(page_url,html)
		print 'mirror ok: '+page_url
	print '[*] mirror finish.'
